{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# First TensorFlow Neurons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(42)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set number of neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_input = 784\n",
    "n_dense = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define placeholder Tensor for simulated MNIST digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, n_input])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Variable Tensors for neuron biases `b` and weight matrix `W`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "b = tf.Variable(tf.zeros([n_dense]))\n",
    "W = tf.Variable(tf.random_uniform([n_input, n_dense])) # 1.\n",
    "# W = tf.Variable(tf.random_normal([n_input, n_dense])) # 2.\n",
    "# W = tf.get_variable('W', [n_input, n_dense], \n",
    "#                     initializer=tf.contrib.layers.xavier_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design the computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z = tf.add(tf.matmul(x, W), b)\n",
    "a = tf.sigmoid(z) # first with tf.sigmoid(), then stick with tf.tanh() or tf.nn.relu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create op for variable initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "initializer_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Execute the graph in a session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as session:\n",
    "    session.run(initializer_op)\n",
    "    \n",
    "    layer_output = session.run(a, {x: np.random.random([1, n_input])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "         1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.]], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADhJJREFUeJzt3H+M5PVdx/HnS64UW1MPuIXgHfWuyVV7Nm1KVoJt0tBi\nFGjTwwQMBNsTL15UxNo2CmgixqQJRGOxSX/kBORqEEoolotSlVAaovXQpVTKj1JOQFg5ua388AeJ\n7eHbP+aL2V72bvbmO7PDfng+ks3OfOczM+/vsTzve9+dmVQVkqR2fd+0B5AkTZahl6TGGXpJapyh\nl6TGGXpJapyhl6TGGXpJapyhl6TGGXpJatyaaQ8AsG7dutq4ceO0x5CkVeXee+/9dlXNDFv3igj9\nxo0bmZubm/YYkrSqJPmX5azz1I0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0k\nNe4V8c5Y6ZVs42V/OZXnfeLK903ledUej+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGG\nXpIaNzT0Sa5Lsj/JA4u2/X6Sbya5P8mfJ1m76LbLk+xN8kiSn57U4JKk5VnOEf31wJkHbbsDeGtV\nvQ34FnA5QJItwPnAj3X3+XSSo8Y2rSTpiA0NfVXdDTx70La/qaoD3dU9wIbu8lbgpqr6n6p6HNgL\nnDrGeSVJR2gc5+h/AfhSd3k98NSi2+a7bZKkKekV+iS/DRwAbnh50xLL6hD33ZFkLsncwsJCnzEk\nSYcxcuiTbAPeD1xYVS/HfB44edGyDcDTS92/qnZW1WxVzc7MzIw6hiRpiJFCn+RM4FLgA1X14qKb\ndgPnJ3ltkk3AZuAf+o8pSRrV0M+jT3IjcDqwLsk8cAWDV9m8FrgjCcCeqvqlqnowyc3AQwxO6Vxc\nVS9NanhJ0nBDQ19VFyyx+drDrP848PE+Q0mSxsd3xkpS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO\n0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS\n4wy9JDXO0EtS4wy9JDXO0EtS44aGPsl1SfYneWDRtuOS3JHk0e77sd32JPlkkr1J7k9yyiSHlyQN\nt5wj+uuBMw/adhlwZ1VtBu7srgOcBWzuvnYAnxnPmJKkUQ0NfVXdDTx70OatwK7u8i7gnEXbP1cD\ne4C1SU4a17CSpCM36jn6E6tqH0D3/YRu+3rgqUXr5rttkqQpGfcvY7PEtlpyYbIjyVySuYWFhTGP\nIUl62aihf+blUzLd9/3d9nng5EXrNgBPL/UAVbWzqmaranZmZmbEMSRJw4wa+t3Atu7yNuC2Rds/\n1L365jTghZdP8UiSpmPNsAVJbgROB9YlmQeuAK4Ebk6yHXgSOK9bfjtwNrAXeBG4aAIzS5KOwNDQ\nV9UFh7jpjCXWFnBx36EkSePjO2MlqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+gl\nqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGG\nXpIaZ+glqXGGXpIa1yv0ST6S5MEkDyS5MckxSTYluSfJo0k+n+TocQ0rSTpyI4c+yXrg14DZqnor\ncBRwPnAV8Imq2gw8B2wfx6CSpNH0PXWzBvj+JGuA1wH7gPcCt3S37wLO6fkckqQeRg59Vf0r8AfA\nkwwC/wJwL/B8VR3ols0D65e6f5IdSeaSzC0sLIw6hiRpiD6nbo4FtgKbgB8CXg+ctcTSWur+VbWz\nqmaranZmZmbUMSRJQ/Q5dfOTwONVtVBV3wVuBd4JrO1O5QBsAJ7uOaMkqYc+oX8SOC3J65IEOAN4\nCLgLOLdbsw24rd+IkqQ++pyjv4fBL12/Bnyje6ydwKXAR5PsBY4Hrh3DnJKkEa0ZvuTQquoK4IqD\nNj8GnNrncSVJ4+M7YyWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn\n6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWp\ncb1Cn2RtkluSfDPJw0l+IslxSe5I8mj3/dhxDStJOnJ9j+j/CPirqvpR4O3Aw8BlwJ1VtRm4s7su\nSZqSkUOf5A3Au4FrAarqO1X1PLAV2NUt2wWc03dISdLo+hzRvwlYAP4kyX1JrknyeuDEqtoH0H0/\nYQxzSpJG1Cf0a4BTgM9U1TuA/+YITtMk2ZFkLsncwsJCjzEkSYfTJ/TzwHxV3dNdv4VB+J9JchJA\n933/Uneuqp1VNVtVszMzMz3GkCQdzsihr6p/A55K8iPdpjOAh4DdwLZu2zbgtl4TSpJ6WdPz/pcA\nNyQ5GngMuIjBXx43J9kOPAmc1/M5JEk99Ap9VX0dmF3ipjP6PK4kaXx8Z6wkNc7QS1LjDL0kNc7Q\nS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1Lj\nDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNa536JMcleS+JH/RXd+U5J4kjyb5fJKj+48p\nSRrVOI7oPww8vOj6VcAnqmoz8BywfQzPIUkaUa/QJ9kAvA+4prse4L3ALd2SXcA5fZ5DktRP3yP6\nq4HfBP63u3488HxVHeiuzwPrez6HJKmHkUOf5P3A/qq6d/HmJZbWIe6/I8lckrmFhYVRx5AkDdHn\niP5dwAeSPAHcxOCUzdXA2iRrujUbgKeXunNV7ayq2aqanZmZ6TGGJOlwRg59VV1eVRuqaiNwPvDl\nqroQuAs4t1u2Dbit95SSpJFN4nX0lwIfTbKXwTn7ayfwHJKkZVozfMlwVfUV4Cvd5ceAU8fxuJKk\n/nxnrCQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMM\nvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuNGDn2S\nk5PcleThJA8m+XC3/bgkdyR5tPt+7PjGlSQdqT5H9AeAj1XVW4DTgIuTbAEuA+6sqs3And11SdKU\njBz6qtpXVV/rLv8n8DCwHtgK7OqW7QLO6TukJGl0YzlHn2Qj8A7gHuDEqtoHg78MgBMOcZ8dSeaS\nzC0sLIxjDEnSEnqHPskPAF8Afr2q/mO596uqnVU1W1WzMzMzfceQJB1Cr9AneQ2DyN9QVbd2m59J\nclJ3+0nA/n4jSpL66POqmwDXAg9X1R8uumk3sK27vA24bfTxJEl9relx33cBHwS+keTr3bbfAq4E\nbk6yHXgSOK/fiJKkPkYOfVX9LZBD3HzGqI8rSRov3xkrSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLU\nOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMv\nSY0z9JLUOEMvSY0z9JLUOEMvSY2bWOiTnJnkkSR7k1w2qeeRJB3eREKf5CjgU8BZwBbggiRbJvFc\nkqTDm9QR/anA3qp6rKq+A9wEbJ3Qc0mSDmNSoV8PPLXo+ny3TZK0wtZM6HGzxLb6ngXJDmBHd/W/\nkjwyoVkmaR3w7WkPscLc5xWSq1b6Gf+f/41Xjx9ezqJJhX4eOHnR9Q3A04sXVNVOYOeEnn9FJJmr\nqtlpz7GS3Of2vdr2F9rf50mduvlHYHOSTUmOBs4Hdk/ouSRJhzGRI/qqOpDkV4G/Bo4CrquqByfx\nXJKkw5vUqRuq6nbg9kk9/ivEqj71NCL3uX2vtv2Fxvc5VTV8lSRp1fIjECSpcYZ+GZbzcQ5JfjbJ\nQ0keTPJnKz3jOA3b3yRvTHJXkvuS3J/k7GnMOU5JrkuyP8kDh7g9ST7Z/Zncn+SUlZ5x3Jaxzxd2\n+3p/kq8meftKzzhuw/Z50bofT/JSknNXaraJqiq/DvPF4JfJ/wy8CTga+Cdgy0FrNgP3Acd210+Y\n9twT3t+dwC93l7cAT0x77jHs97uBU4AHDnH72cCXGLxH5DTgnmnPvAL7/M5FP9NnvRr2uVtzFPBl\nBr9jPHfaM4/jyyP64ZbzcQ6/CHyqqp4DqKr9KzzjOC1nfwt4Q3f5BznoPRKrUVXdDTx7mCVbgc/V\nwB5gbZKTVma6yRi2z1X11Zd/poE9DN4Ps6ot478zwCXAF4DV/P/x9zD0wy3n4xzeDLw5yd8l2ZPk\nzBWbbvyWs7+/C/xcknkGRz2XrMxoU/Vq/1iP7Qz+RdO0JOuBnwE+O+1ZxsnQDzf04xwYvEx1M3A6\ncAFwTZK1E55rUpazvxcA11fVBganNP40Ses/S8v5c2lSkvcwCP2l055lBVwNXFpVL017kHGa2Ovo\nGzL04xy6NXuq6rvA493n9mxm8A7h1WY5+7sdOBOgqv4+yTEMPiukmX/qLmE5fy7NSfI24BrgrKr6\n92nPswJmgZuSwOBn+uwkB6rqi9Mdq5/Wj8LGYTkf5/BF4D0ASdYxOJXz2IpOOT7L2d8ngTMAkrwF\nOAZYWNEpV95u4EPdq29OA16oqn3THmqSkrwRuBX4YFV9a9rzrISq2lRVG6tqI3AL8CurPfLgEf1Q\ndYiPc0jye8BcVe3ubvupJA8BLwG/sVqPfpa5vx8D/jjJRxicvvj56l6usFoluZHBqbd13e8ergBe\nA1BVn2Xwu4izgb3Ai8BF05l0fJaxz78DHA98ujvCPVCr/IO/lrHPTfKdsZLUOE/dSFLjDL0kNc7Q\nS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNe7/APDkU3rN3fHLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb3307bd470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(np.transpose(layer_output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
